import java.lang.StringBuffer;
import java.util.Scanner;
import java.util.regex.Pattern;
import java.io.*;
import java.net.*;

/*
 * class to save a web pages html to a string buffer
 */
public class HTMLRetriever 
{
    Scanner scanner;
    
    public HTMLRetriever () 
    {
    }


    /*
     * This method opens a connection to a website,
     * and saves the html source to a stringbuffer
     * for further use.
     */
	public StringBuffer getHTML (String url) 
	{
	    StringBuffer buff = new StringBuffer("");
	    String line;
	    URL link;
		try 
		{
			link = new URL(url);
			try 
			{
				DataInputStream source = new DataInputStream(link.openStream());
				
				try 
				{
					while ((line = source.readLine()) != null) 
					{
						buff.append(line);
					}
				}
				catch (Exception e)
				{
					System.err.println("there's a problem adding a line "+e);
				}
			}
			catch (IOException e)
			{
				System.err.println(e);
			}
		}
		catch (Exception e)
		{
			System.err.println(e);
		}
		
		removeAccents(buff);
		
		return buff;
	}
	/*
	 * TODO Optimise regexes eg &#2(3(2|3))
	 */
	public StringBuffer removeAccents(StringBuffer buff)
	{
		
		String str = buff.toString();
		
		str = Pattern.compile("\u00C0|\u00C1|\u00C2|\u00C3|\u00C4\u00C5|\u00E0|\u00E1|\u00E2|\u00E3|\u00E4|\u00E5|&#a(\\w)+;|&#192;|&#224;|&#193;|&#225;|&#194;|&#226;|&#195;|&#227;|&#196;|&#228;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("a");//a's
		str = Pattern.compile("\u00C8|\u00C9|\u00CA|\u00CB|\u00E8|\u00E9|\u00EA|\u00EB|&#e(\\w)+;|&#200;|&#232;|&#201;|&#233;|&#202;|&#234;|&#203;|&#235;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("e");//e's
		str = Pattern.compile("\u00EC|\u00ED|\u00EE|\u00EF|\u00CC|\u00CD|\u00CE|\u00CF|&#i(\\w)+;|&#204;|&#236;|&#205;|&#237;|&#206;|&#238;|&#207;|&#239;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("i");//i's
		str = Pattern.compile("\u00D2|\u00D3|\u00D4|\u00D5|\u00D6|\u00D8|\u00F2|\u00F3|\u00F4|\u00F5|\u00F6|\u00F8|&#o(\\w)+;|&#210;|&#242;|&#211;|&#243;|&#212;|&#244;|&#213;|&#245;|&#214;|&#246;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("o");//o's
		str = Pattern.compile("\u00D9|\u00DA|\u00DB|\u00DC|\u00F9|\u00FA|\u00FB|\u00FC|&u(\\w);|&#217;|&#249;|&#218;|&#250;|&#219;|&#251;|&#220;|&#252;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("u");//u's
		str = Pattern.compile("\u00DD|\u0178|\u00FD|\u00FF|&y(\\w)+;|&#221;|&#253;|&#159;|&#255;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("y");//y's
		str = Pattern.compile("\u00F1|\u00D1|&#209;|&n(\\w)+;|&#241;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("n");//n's
		str = Pattern.compile("\u00C6|\u00E6|&#198;|&aelig;|&#230;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("ae");//ae's
		str = Pattern.compile("\u00E7|\u00C7|&Ccedil;|&#199;|&ccedil;|&#231;", Pattern.CASE_INSENSITIVE).matcher(str).replaceAll("c");//c's
		str = Pattern.compile("\'").matcher(str).replaceAll(" ");//remove apostrophes
		
		StringBuffer sb = new StringBuffer(str);
		return sb;
	}
}
